const request=require("superagent");
const cheerio = require('cheerio');
const iconv = require('iconv-lite');
const util = require('util');
const charset = require("superagent-charset")(request);
const fs=require('fs');
var mysql      = require('mysql');
const assert = require('assert');
var connection = mysql.createConnection({
  host     : '127.0.0.1',
  user     : 'root',
  password : 'f9fe8cd75990fffe',
  database : 'a.com'
});
connection.query = util.promisify(connection.query);
async function getResourceTree(page) {
    var resource = await page._client.send('Page.getResourceTree');
    return resource.frameTree;
}

async function getResourceContent(page, url) {
    const { content, base64Encoded } = await page._client.send(
        'Page.getResourceContent',
        { frameId: String(page.mainFrame()._id), url },
    );
    assert.equal(base64Encoded, true);
    return content;
};
const Agent=[
 	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",
    "Mozilla/5.0 (Linux; U; Android 2.3.7; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "MQQBrowser/26 Mozilla/5.0 (Linux; U; Android 2.3.7; zh-cn; MB200 Build/GRJ22; CyanogenMod-7) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1",
    "JUC (Linux; U; 2.3.7; zh-cn; MB200; 320*480) UCWEB7.9.3.103/139/999",
    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:7.0a1) Gecko/20110623 Firefox/7.0a1 Fennec/7.0a1",
    "Opera/9.80 (Android 2.3.4; Linux; Opera Mobi/build-1107180945; U; en-GB) Presto/2.8.149 Version/11.10",
    "Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13",
    "Mozilla/5.0 (iPhone; U; CPU iPhone OS 3_0 like Mac OS X; en-us) AppleWebKit/420.1 (KHTML, like Gecko) Version/3.0 Mobile/1A542a Safari/419.3",
    "Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_0 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8A293 Safari/6531.22.7",
    "Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B334b Safari/531.21.10",
    "Mozilla/5.0 (BlackBerry; U; BlackBerry 9800; en) AppleWebKit/534.1+ (KHTML, like Gecko) Version/6.0.0.337 Mobile Safari/534.1+",
    "Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.0; U; en-US) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/233.70 Safari/534.6 TouchPad/1.0",
    "Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/20.0.019; Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.18124",
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0; HTC; Titan)",
    "Mozilla/5.0 (Linux; U; Android 8.1.0; zh-cn; BLA-AL00 Build/HUAWEIBLA-AL00) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/57.0.2987.132 MQQBrowser/8.9 Mobile Safari/537.36",
];
const timeout={
	response:50000,
	deadline:600000,
}
Date.prototype.format = function (fmt) {
    var o = {
        "M+": this.getMonth() + 1, //月份
        "d+": this.getDate(), //日
        "h+": this.getHours(), //小时
        "m+": this.getMinutes(), //分
        "s+": this.getSeconds(), //秒
        "q+": Math.floor((this.getMonth() + 3) / 3), //季度
        "S": this.getMilliseconds() //毫秒
    };
    if (/(y+)/.test(fmt)) fmt = fmt.replace(RegExp.$1, (this.getFullYear() + "").substr(4 - RegExp.$1.length));
    for (var k in o)
        if (new RegExp("(" + k + ")").test(fmt)) fmt = fmt.replace(RegExp.$1, (RegExp.$1.length == 1) ? (o[k]) : (("00" + o[k]).substr(("" + o[k]).length)));
    return fmt;
}
function delay(t){
    return new Promise(resolve => setTimeout(resolve, t))
}

function rnd(n, m){
    var random = Math.floor(Math.random()*(m-n+1)+n);
    return random;
}
function getRndIP(){
    var ip=[];
    for(var i=0;i<4;i++){
    ip.push(rnd(10,250));
    }
    return ip.join(".");
}
async function getList(){
	var ip=getRndIP();
		var agent=Agent[rnd(0,Agent.length-1)];
		agent=Agent[0];
		var option={
			"Host":"www.jykxyj.com",
			"User-Agent":agent,
			"Referer":'http://www.jykxyj.com/',
			"X-Forwarded-For":ip,
		}
		let url=`http://www.jykxyj.com/`;
		console.log(url)
		let res=await request.get(url).set(option).timeout(timeout).buffer(true);
		await delay(2000);
		let body=res.text;
		var $ = cheerio.load(body);
		$('#newsList852 .fk-newsListTitle').each(async function(){
			let arr={
					link:$(this).attr('href')
				};

			await getArticle(arr)

		})
}
async function getArticle(arr){
	var ip=getRndIP();
		var agent=Agent[rnd(0,Agent.length-1)];
		agent=Agent[0];
		var option={
			"Host":"www.jykxyj.com",
			"User-Agent":agent,
			"Referer":'http://www.jykxyj.com/',
			"X-Forwarded-For":ip,
		}
		
		if(arr.link.indexOf('http://')==-1){
			arr.link='http://www.jykxyj.com/'+arr.link
		}
		
	let url=arr.link;
	let res=await request.get(url).set(option).timeout(timeout).buffer(true);
	await delay(2000);
	let body=res.text;
	var $ = cheerio.load(body);
	
	let list={
				title:$('h1').text(),
				content:$('.richContent').html(),
				article_cate_id:14
			};
	console.log(list.title)
	addDb(list);
}

//插入数据库
async function addDb(data){
	// console.log(data.nameId)
	let sql=`select * from tplay_article where title='${data.title}'`;
	let result=await connection.query(sql);
	if(result.length>0){
		console.log(`add ${data.title} error`);
	}else{
		//插入数据
		let addSql="INSERT INTO tplay_article SET  ?";
		let addSqlParams = data;
		await connection.query(addSql,addSqlParams,function (err, result) {
	        if(err){
	         console.log('[INSERT ERROR] - ',err.message);
	         return;
	        }        
	 		console.log('INSERT ID:',result.insertId);
		       
		});
		
	}
	
		
}
getList();
